
package ma.crawler.web;

import android.text.TextUtils;

import ma.http.HttpUtils;
import ma.util.FileUtil;
import ma.util.Log;

import java.io.File;
import java.util.LinkedList;
import java.util.Queue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class WebCrawler {

    public static class Config {
        
    }

    public static void main(String[] args) {
        String webUrl = "http://www.yayu.org/book/gnu_make/index.html";
        Queue<String> queue = new LinkedList<String>();
        queue.add(webUrl);
        while (!queue.isEmpty()) {
            String url = queue.poll();
            if (!existUrl(url)) {
                String html = HttpUtils.getUrlAsString(url);
                if (html != null) {
                    File file = getFileByUrl(url);
                    FileUtil.saveToFile(file, html, "GB2312");
                    Log.d("TAG", "URL:" + url);
                    String currentPath = getCurrentPath(url);
                    parseUrl(currentPath, html, queue);
                    downloadImg(currentPath, html);
                } else {
                    Log.e("TAG", "null:" + url);
                }
            }
        }
    }

    private static void parseUrl(String currentPath, String html, Queue<String> queue) {
        String linkReg = "<a\\s.*?href=\"([^\"]+)\"[^>]*>(.*?)</a>";
        Pattern pattern = Pattern.compile(linkReg);
        Matcher matcher = pattern.matcher(html);
        while (matcher.find()) {
            String url = matcher.group(1);
            if (isEmailLink(url)) {
                continue;
            }
            if (url.startsWith("#")) {
                continue;
            }
            if (url.contains("#")) {
                url = url.substring(0, url.lastIndexOf('#'));
            }
            if (!url.startsWith("http:/")) {
                url = currentPath + url;
            }
            if (isInBlackList(url)) {
                continue;
            }
            if (!queue.contains(url)) {
                queue.add(url);
            }
        }
    }

    private static void downloadImg(String currentPath, String html) {
        String imgReg = "<img[^>]*src=\"([^\"]+)\">";
        Pattern pattern = Pattern.compile(imgReg);
        Matcher matcher = pattern.matcher(html);
        while (matcher.find()) {
            String url = matcher.group(1);
            if (!url.startsWith("http:/")) {
                url = currentPath + url;
            }
            if (isInBlackList(url)) {
                continue;
            }
            HttpUtils.downloadFile(url, getFileByUrl(url));
        }
    }

    private static boolean existUrl(String url) {
        return getFileByUrl(url).exists();
    }

    private static File getFileByUrl(String url) {
        return new File(url.replace("http:/", ""));
    }

    private static String getCurrentPath(String url) {
        int endIndex = url.lastIndexOf('/') + 1;
        return url.substring(0, endIndex);
    }

    private static boolean isEmailLink(String url) {
        return !TextUtils.isEmpty(url) && url.startsWith("mailto:");
    }

    private static boolean isInBlackList(String url) {
        return url.equals("http://xhbdahai.cublog.cn");
    }

}
